/**
 * This file contains code common to any Rio back end.
 * TODO Split these features, maybe after the compiler is in Rio.
 * TODO Just that this file is getting big (941 lines when typed this comment).
 */
(function(){


var global = this;

// Namespace.

var rio = global.rio;
if (!rio) {
  rio = global.rio = {};
}


// Exports.

// TODO Change to eachIndex and each, respectively?
rio.each = each;
rio.eachField = eachField;
rio.ensureLibs = ensureLibs;
rio.extract = extract;
rio.listLibs = listLibs;
rio.log = log;
rio.noop = noop;
rio.parse = parse;
rio.resolve = resolve;
rio.Symbol = Symbol;


// Exported node types.


// Token nodes (which might as be used as tree nodes).

// Standard doesn't require exposing function names.
// So make them explicit.
var And = Symbol('And');
var Comma = Symbol('Comma');
var Comment = Symbol('Comment');
var Dedent = Symbol('Dedent');
var Def = Symbol('Def');
var End = Symbol('End');
var LineFeed = Symbol('LineFeed');
var Equals = Symbol('Equals');
var Err = Symbol('Err');
var Id = Symbol('Id');
var Indent = Symbol('Indent');
var LParen = Symbol('LParen');
var Minus = Symbol('Minus');
var Not = Symbol('Not');
var Num = Symbol('Num');
var Or = Symbol('Or');
var Plus = Symbol('Plus');
var Put = Symbol('Put');
var RParen = Symbol('RParen');
var Semi = Symbol('Semi');
var Star = Symbol('Star');
var Str = Symbol('Str');
var Var = Symbol('Var');
var White = Symbol('White');


// Tree nodes (never corresponding to tokens).

var Args = Symbol('Args');
var Block = Symbol('Block');
var Call = Symbol('Call');
var Group = Symbol('Group');
var Param = Symbol('Param');
var Params = Symbol('Params');


// Keywords.

var keys = {
  def: Def,
  end: End,
  // TODO Change 'var' to 'let' or 'val'? And 'trailing!' for non-const?
  // TODO Maybe use ':=' (vs. ': Type ='), but that makes vars shorter than
  // TODO defs, and I don't like those economics.
  'var': Var
};


// Operators.

var ops = {
  '&': And,
  ',': Comma,
  '==': Equals,
  '(': LParen, // TODO ParenL and ParenR?
  '-': Minus,
  '~': Not,
  '|': Or,
  '+': Plus,
  '=': Put,
  ')': RParen,
  ';': Semi,
  '*': Star
};


// Error types.

// Helper function for building errors.
function Error(type, node) {
  var error = {type: type, node: node, pos: nodePos(node)};
  node.error = error;
  return error;
}

var Unresolved = Symbol('Unresolved');


// Function definitions.

// Creates a default context of core Rio features.
// Code generation and function implementation is platform specific.
// Only the signatures, including types are defined here.
// The context is a map from id to array of implementations.
function defaultContext() {
  // TODO Finalize the representation of function signatures.
  // TODO Move these out to a separate rio file?
  // TODO If so, would need a build process.
  var defs = [{
    // TODO Plus or add? Implications of side effects?
    name: 'add',
    sig: [['Number+'], ['Number']]
  }, {
    name: 'and',
    sig: [['Boolean+'], ['Boolean']]
  }, {
    name: 'Boolean',
    sig: [['Type']]
  }, {
    name: 'equals',
    sig: [['Any', 'Any'], ['Boolean']]
  }, {
    name: 'false',
    sig: [['Boolean']]
  }, {
    name: 'if',
    params: 'Result',
    sig: [
      ['Boolean', [['Result']]],
      'else', [['Result']],
      ['Result']
    ]
  }, {
    name: 'log',
    sig: [['String'], []]
  }, {
    name: 'multiply',
    sig: [['Number+'], ['Number']]
  }, {
    // TODO Support element-wise not, and/or set complements?
    name: 'not',
    sig: [['Boolean'], ['Boolean']]
  }, {
    // TODO How do not (set complement?), negative (additive inverse), and inverse (multiplicative inverse) relate?
    // TODO Negative rather than negate? Implications of side effects?
    name: 'negate',
    sig: [['Number'], ['Number']]
  }, {
    name: 'or',
    sig: [['Boolean+'], ['Boolean']]
  }, {
    // TODO Should we parse return specially or is this good enough?
    // TODO Depends on how we parse or allow 'return if ...'. Where do the blocks bind?
    name: 'return',
    sig: [['Any']]
  }, {
    name: 'String',
    sig: [['Type']]
  }, {
    name: 'true',
    sig: [['Boolean']]
  }, {
    name: 'Type',
    sig: [['Type']]
  }];
  var context = {};
  for (var d in defs) {
    var def = defs[d];
    def.ns = 'core';
    // TODO If we have overloaded defs in default, then append.
    context[def.name] = [def];
  }
  return context;
}

function each(items, handler) {
  if (items) {
    for (var i = 0; i < items.length; i++) {
      handler(items[i], i);
    }
  }
}

function eachField(object, handler) {
  if (object) {
    for (var key in object) {
      handler(object[key], key);
    }
  }
}

function ensureLibs(script, handler) {
  // TODO Actually ensure the libs are listed -- and loaded if needed.
  // TODO Will need some kinds of access to recursive compiling services.
  var listTotal = script.env.libs.length;
  if (!listTotal) {
    handler();
    return;
  }
  // TODO Just fork off the compiles and join later.
  // TODO This is too complicated here and doesn't allow for circular refs.
  var listedCount = 0;
  var compileTotal = 0;
  var compiledCount = 0;
  each(script.env.libs, function(lib) {
    if (lib.listed) {
      checkLoad();
    } else {
      pushTo(lib, 'onLists', checkLoad);
    }
    function checkLoad() {
      listedCount++;
      eachField(script.refs, function(refs, id) {
        var export = lib.exports[id];
        if (export) {
          // log('Found export for refs ' + id);
          get(script, 'imports')[id] = {};
          eachField(export, function(exportScript) {
            // TODO Import of rio.compile won't work out there unless we defer lib execution.
            // TODO Support circular refs before conversion to Rio?
            if (!exportScript.env) {
              // TODO Do better!
              exportScript.env = {defs: {}, libs: []};
            }
            compileTotal++;
            rio.compile(exportScript, function() {
              // Put the relevant exports in a convenient place.
              eachField(exportScript.exports, function(exportScriptExport) {
                // TODO Check override priority by env libs order.
                if (exportScriptExport.name === id) {
                  var import = get(get(script, 'imports'), id);
                  import.name = id;
                  eachField(exportScriptExport.defs, function(def, name) {
                    get(import, 'defs')[name] = def;
                  });
                }
              });
              // Now track our compilation progress.
              compiledCount++;
              if (listedCount === listTotal && compiledCount === compileTotal) {
                // We've listed all the libs and compiled all the script we want from them.
                handler();
              }
            });
          });
        }
      });
      if (listedCount === listTotal && !compileTotal) {
        // We've listed all the libs and nothing to compile.
        handler();
      }
    }
  });
}

function extract(script) {
  if (!script.tree.context) {
    script.tree.context = defaultContext();
  }
  var context = script.tree.context;

  var refs = script.refs = {};

  extractExpr(script.tree);

  // TODO Combine with extractBlock as extractExprs?
  function extractArgs(node) {
    extractBlock(node);
  }

  function extractBlock(node) {
    each(node.expr, function(kid) {
      extractExpr(kid);
    });
  }

  function extractCall(node) {
    pushTo(get(refs, node.id[0].text), 'nodes', node);
    var args = node.args;
    for (var i = 0; i < args.length; i++) {
      extractArgs(args[i]);
    }
  }

  function extractDef(node) {
    // Store the def in the parent scope.
    var id = node.id[0].text;
    // log("def'd " + id);
    // TODO Standardize this parent access.
    var scope = node.parent.scope || (node.parent.scope = {});
    scope[id] = node;
    // Add parameters at a scope of the def.
    var defScope = node.scope || (node.scope = {});
    var params = node.params[0];
    // TODO Change 'expr' to 'param'?
    each(node.params[0].expr, function(param, p) {
      defScope[param.id[0].text] = param;
      // log("param'd " + param.id[0].text);
    });
    // TODO Multidefs.
    extractExpr(node.expr[0]);
  }

  function extractExpr(node) {
    switch(node.type) {
      case Block:
        extractBlock(node);
        break;
      case Call:
        extractCall(node);
        break;
      case Def:
        extractDef(node);
        break;
      case Var:
        extractVar(node);
        break;
      default:
    }
  }

  function extractVar(node) {
    // TODO Merge some logic with extractDef
    var id = node.id[0].text;
    // log("var'd " + id);
    // TODO Standardize this parent access.
    var scope = node.parent.scope || (node.parent.scope = {});
    scope[id] = node;
    // TODO How to represent the 'put' side of things? -> put(it, @id, ...)
    if (node.expr) {
      extractExpr(node.expr[0]);
    }
  }

}

/**
 * Creates the field if it doesn't exist, then returns it.
 */
function get(object, id) {
  return object[id] || (object[id] = {});
}

function nodePos(node) {
  return traverseNode(node, function(node) {
    if (node.pos) {
      return node.pos;
    }
  });
}

function listLibs(script) {
  var env = script.env;
  each(env.libs, function(lib) {
    if (!(lib.listed || lib.listing)) {
      lib.listing = true;
      // TODO Handle error case to allow retries??? Or is that trouble?
      rio.listLib(lib.uri, function(scripts) {
        lib.listing = false;
        each(scripts, function(script) {
          script.lib = lib;
          // TODO !!! -> Store by script keys to avoid duping??
          eachField(script.exports, function(export) {
            // Store in general env and in lib, for easy access in different cases.
            get(get(env, 'exports'), export.name)[script.uri] = script;
            get(get(lib, 'exports'), export.name)[script.uri] = script;
          });
        });
        lib.listed = true;
        each(lib.onLists, function(onList) {onList(lib)});
      });
    }
  });
}

function log(message) {
  // return; // Skip logging for the moment.
  // Depends on Firebug API.
  if (global.console) {
    global.console.log(message);
  }
}

function matchSplit(string, regexp, handle) {
  var delims = string.match(regexp) || [];
  var middles = string.split(regexp);
  for (var d = 0; d < delims.length; d++) {
    handle(middles[d]);
    handle(delims[d]);
  }
  handle(middles[delims.length]);
}

/**
 * Simple "do nothing" that's sometimes handy.
 */
function noop() {}

/**
 * TODO Watcher support for incremental handling.
 * @returns a parsed tree
 */
function parse(script) {

  var text = script.text;
  var tokens = tokenize(text);

  var root = {type: Block};
  script.errors = [];
  var stack = [root];
  var current = root;
  var t = -1;
  var token = null;
  next();
  parseBlock();
  // log(tokens);

  script.tree = root;
  return;

  function inBlock() {
    return current.parent.type === Block;
  }

  function more() {
    return t < tokens.length;
  }

  function namedNode(name, type, builder) {
    // TODO Eventually make this the only way.
    // TODO And clean it up a little.
    var field = current[name];
    if (!field) {
      current[name] = field = [];
    }
    if (builder) {
      var kid = {type: type};
      field.push(kid);
      var parent = kid.parent = current;
      current = kid;
      stack.push(current);
      try {
        builder();
      } finally {
        current = parent;
        stack.pop();
      }
    } else {
      // Not really a type. Probably a token.
      field.push(type);
    }
  }

  function next() {
    do {
      token = more() ? tokens[++t] : null;
    } while (token && (token.type === White || token.type === Comment));
    return token;
  }

  function node(type, builder) {
    namedNode('expr', type, builder);
  }

  function parseArgs() {
    // Paren only counts if it's the next token! Whitespace makes it an expr!
    var doParenArgs = more() && tokens[t+1].type == LParen;
    next();
    // if lparen, then parseParenArgs
    // else ...
    // TODO Ideally peek(parseExpr); or something.
    // TODO Maybe even a "looks like Expr" token thing.
    switch (token.type) {
      // TODO Can we say things like 'say sin pi/2' or 'go to (3,4)' or 'goTo 3 4'???
      // TODO For now, the first one is the interpretation.
      case LParen:
        if (doParenArgs) {
          parseParenArgs();
          break;
        }
      // TODO Standardize how to start an expr.
      case Id:
      case Minus:
      case Not:
      case Num:
      case Str:
        if (!inBlock()) {
          // TODO Error for non-paren except at block level?
        }
        parseExpr();
        break;
      default:
        // TODO Not lparen or expr.
        // TODO Let errors be handled elsewhere?
        // TODO Or customize better error message here?
    }
    // TODO Should be able to skip multiple no-op lines.
    if (token.type === LineFeed) {
      for (var s = stack.length - 2; s >= 0; s--) {
        var type = stack[s].type;
        if (type === Args) {
          // Let the outer args have it.
          return;
        } else if (type !== Call) {
          // TODO But what about if a + b > c ...???
          // TODO Maybe better to list bad (parens, assign, ...) than good.
          break;
        }
      }
      // TODO Return out through Args and Call to the outermost such Args.
      // TODO Except don't go through Parens.
      // TODO That's where this belongs.
      var peeked = peek();
      if (peeked && peeked.type === Indent) {
        stack[stack.length-2].multi = true;
        next();
        next();
        node(Block, parseBlock);
      }
    }
  }

  function parseBlock() {
    while (more()) {
      parseExpr();
      var del = false;
      while (!del && more()) {
        switch (token.type) {
          case Dedent:
            // Block ends on dedent.
            // TODO Consider block ends on End.
            next();
            return;
          case LineFeed:
          case Semi:
            next();
            del = true;
            break;
          // TODO case End:
          //   ...
          default:
            // error, or can one expr follow another w/o delimiters?
            next();
            break;
        }
      }
    }
  }

  // A direct Id reference here.
  // TODO Some other method could handle nested expression called with official ().
  function parseCall() {
    node(Call, function() {
      namedNode('id', token);
      namedNode('args', Args, parseArgs);
      // TODO Inline multiblocks after '()', next lines after '...'
      if (current.multi) {
        while (more()) {
          switch(token.type) {
            case Dedent:
            case End:
              // TODO Consume or not?
              // All done with the implicit multiblock.
              return;
            case LineFeed:
              // No-op.
              next();
              break;
            case Id:
              namedNode('id', token);
              namedNode('args', Args, parseArgs);
              break;
            default:
              // TODO Error.
              next();
              break;
          }
        }
      }
    });
  }

  function parseDef() {
    node(Def, function() {
      // TODO Multidefs?
      // node(token); // What's best to preserve tokens? Always keep ordered at leaves, and use named fields for parsed meaning?
      next();
      namedNode('id', token);
      next();
      namedNode('params', Params, parseParams);
      // TODO Tree by named fields rather than array indexes.
      // TODO Start with the following (?):
      // TODO Return type, annotations, multis.
      if (token.type !== LineFeed) {
        // TODO Error
        // TODO How draconian do we want to be on formatting?
        // TODO Seek LineFeed?
        return;
      }
      next();
      if (token.type === Indent) {
        next();
        node(Block, parseBlock);
        if (token.type === End) {
          next();
        } else {
          // TODO Error
          // TODO Allow multi-blocks defs or anything????
        }
      }
    });
  }

  function parseExpr() {
    parseInfix(
      And, 'and', parseInfix(
      Or, 'or', parseInfix(
      // TODO Don't allow repeats on equals? Or at least generate differently.
      Equals, 'equals', parseInfix(
      Plus, 'add', parseInfix(
      Star, 'multiply', parseExprCore
    )))))();
  }

  function parseExprCore() {
    // TODO Need outer layer of lower-priority binary ops.
    switch (token.type) {
      case Def:
        parseDef();
        break;
      case Err:
        next();
        break;
      case Id:
        parseCall();
        break;
      case LParen:
        // TODO Distinguish lists of one from non-lists?
        node(Group, parseParenArgs);
        break;
      case Minus:
        parsePrefixOp('negate');
        break;
      case Not:
        parsePrefixOp('not');
        break;
      case Num:
        node(token);
        next();
        break;
      case Str:
        // TODO Parse string interpolation?
        node(token);
        next();
        break;
      case Var:
        parseVar();
        break;
      default:
        // TODO Mark error.
        break;
    }
  }

  function parseInfix(tokenType, functionName, parseDeeper) {
    return function() {
      parseDeeper();
      if (token.type == tokenType) {
        // Wrap what we just parsed into a call to 'functionName'.
        var previous = current.expr.pop();
        node(Call, function() {
          namedNode('id', {text: functionName});
          namedNode('args', Args, function() {
            previous.parent = current;
            current.expr = [previous];
            do {
              next();
              parseDeeper();
            } while (token.type == tokenType);
          });
        });
      }
    };
  }

  function parseListContents() {
    // TODO Unify with block parsing?
    parseListContentsGeneric(parseExpr);
  }

  function parseListContentsGeneric(handler) {
    // TODO Unify with block parsing?
    while (true) {
      switch (token.type) {
        case Comma:
          // TODO Error?
          next();
          continue;
        case Semi:
        case LineFeed:
          // TODO 2D handling?
          next();
          continue;
        case RParen:
          return;
        default:
          handler();
          // TODO Really continue? We don't care about commas or other delimiters?
          // TODO If changed to break, do we need to remove the 'next()' down there?
          continue;
      }
      next();
      // TODO Semis for 2D (even ragged) lists
      if (token.type === Comma && token.type === Semi) {
        next();
      } else {
        if (token.type === RParen) {
          return;
        }
        // TODO Error
      }
    }
  }

  function parseParams() {
    if (token.type !== LParen) {
      // Let someone else deal with it.
      return;
    }
    next();
    parseListContentsGeneric(function() {
      if (token.type === Id) {
        // TODO Need types and annotations.
        // TODO Should name these 'param' instead of default 'expr'?
        node(Param, function() {namedNode('id', token)});
      } else {
        // TODO Error!
      }
      next();
    });
    next(); // Skip RParen.
  }

  function parseParenArgs() {
    // Skip paren.
    next();
    parseListContents();
    if (token.type === RParen) {
      next();
    } else {
      // TODO Error
    }
  }

  function parsePrefixOp(id) {
      // TODO Store the operator token in the tree somewhere?
      next();
      node(Call, function() {
        namedNode('id', {text: id});
        namedNode('args', Args, parseExprCore);
      });
  }

  function parseVar() {
    node(Var, function() {
      // TODO Check if it really i
      var id = next();
      if (id.type !== Id) {
        // TODO Error!
      }
      namedNode('id', id);
      next();
      if (token.type === Id) {
        // TODO parseType (and other metadata?)
        namedNode('datatype', token);
        next();
      }
      if (token.type === Put) {
        next();
        // TODO I'll need to keep a hovering name around.
        // TODO Or I'll need to create a node here to wrap the value.
        // TODO Otherwise I can't name the node.
        parseExpr();
      } else {
        // TODO Error checking?
      }
    });
  }

  /**
   * Consider potential callback peeker that can reset to old state.
   */
  function peek() {
    var p = t;
    while (++p < tokens.length) {
      var peeked = tokens[p];
      if (peeked.type !== White) {
        return peeked;
      }
    }
    return null;
  }

}

/**
 * Creates the field as an array, if it doesn't exist, then pushes the item on.
 */
function pushTo(object, id, item) {
  (object[id] || (object[id] = [])).push(item);
}

// Then resolves references (and later types) against the context.
// Leaves metadata around the tree.
function resolve(script) {
  var context = script.tree.context;
  resolveExpr(script.tree);

  // Function definitions.

  function error(type, node) {
    script.errors.push(Error(type, node));
  }

  function findDef(node, id) {
    if (node.scope) {
      // TODO Do local defs outweigh top-levels, or are they peers?
      var def = node.scope[id];
      if (def) return def;
    }
    if (node.parent) {
      return findDef(node.parent, id);
    }
    return context[id];
  }

  function resolveArgs(node) {
    resolveBlock(node);
  }

  function resolveBlock(node) {
    each(node.expr, function(kid) {
      resolveExpr(kid);
    });
  }

  function resolveCall(node) {
    // TODO Verify that arg numbers and types match.
    // TODO IDs in context can be qualified by '::id' without implying namespace?
    // Loop through all multicall parts (id/expr, args, id, args ...).
    var ids = node.id;
    for (var i = 0; i < ids.length; i++) {
      var id = ids[i].text;
      if (i === 0) {
        // TODO Verify that all IDs match for multicalls, not just the first.
        // TODO Check for IDs in scope before context!!!
        node.def = findDef(node, id);
        if (!node.def) {
          error(Unresolved, node);
        }
      }
      resolveArgs(node.args[i]);
    }
  }

  function resolveDef(node) {
    // TODO Go inside params (for default values ...) and metadata.
    resolveExpr(node.expr[0]);
  }

  function resolveExpr(node) {
    switch(node.type) {
      case Block:
      case Group:
        resolveBlock(node);
        break;
      case Call:
        resolveCall(node);
        break;
      case Def:
        resolveDef(node);
        break;
      case Var:
        resolveVar(node);
        break;
      default:
    }
  }

  function resolveVar(node) {
    // TODO Resolve types and metadata.
    if (node.expr) {
      resolveExpr(node.expr[0]);
    }
  }

}

/**
 * TODO Make it treat sub as raw, not a regex!
 */
function startsWith(string, sub) {
  return string.match('^' + sub);
}

function testFull(regexp, string) {
  var matches = regexp.exec(string);
  return matches && matches[0].length == string.length;
}

function Symbol(name) {
  // Store objects with name instead of strings directly.
  // That way '===' will be faster and more precise.
  // TODO Or should I just do strings?
  var Symbol = {name: name};
  // Export all node types.
  rio[name] = Symbol;
  return Symbol;
}

function tokenize(text, watch) {

  var tokens = [];
  var buffer, colNumber, finishSub, handleSub, newLine, quote, startCol;
  var indents = [''];
  var lineNumber = 0;
  var lines = text.split(/\r?\n/);
  for (var L = 0; L < lines.length; L++) {
    var line = lines[L];
    lineNumber++;
    // Set up initial state.
    buffer = null;
    colNumber = 1;
    finishSub = noop;
    handleSub = handleCode;
    quote = '';
    // Handle tokens at code, comment, string level.
    // log(lineNumber + ": " + line);
    // Check for indent level first. Just easier that way.
    line = updateIndents(line);
    matchSplit(line, /#|\\.|"/g, handle);
    finish();
    // TODO Retain original chars.
    token(LineFeed, "\n", colNumber);
  }
  return tokens;

  function append(string) {
    if (!buffer) {
      buffer = [];
      startCol = colNumber;
    }
    buffer.push(string);
  }

  function changeHandler(handler, finisher) {
    finish();
    handleSub = handler;
    finishSub = finisher;
  }
  
  function drain() {
    content = buffer ? buffer.join('') : '';
    buffer = null;
    return content;
  }

  function finish() {
    finishSub();
    finishSub = noop;
  }

  function finishComment() {
    var content = drain();
    token(Comment, content, startCol);
  }

  function finishString() {
    var content = drain();
    // TODO Convert escapes to official string contents. Use regexps, probably.
    token(Str, content, startCol);
  }

  function handle(content) {
    if (content) {
      handleSub(content);
    }
    colNumber += content.length;
  }

  function handleCode(content) {
    if (content == '#') {
      changeHandler(handleComment, finishComment);
    } else if (content == '"') {
      changeHandler(handleString, finishString);
      quote = content;
    } else {
      // TODO Break it down more.
      var subCol = colNumber;
      // TODO Automatically construct this regexp from list of opsyms.
      // TODO Or split on keyword pattern? More general? Slower?
      matchSplit(content, /\s+|;|==?|\(|\)|,|~|\*|\+|-|&|\|/g, function(part) {
        // log(part);
        if (/\s/.test(part)) {
          // Whitespace matters in Rio, but only for indents???
          token(White, part, subCol);
        } else if (testFull(/[A-Za-z_][A-Za-z0-9_]*/, part)) {
          token(keys[part] || Id, part, subCol);
        } else if (testFull(/[0-9]+(?:\.[0-9]+)?/, part)) {
          // Numbers must have leading digits, and if they have a dot, they must have trailing.
          // TODO Good error reporting for identifiable bad number literals?
          // TODO Exponent notation.
          token(Num, part, subCol);
        } else if (part) {
          var opType = ops[part] || Err;
          token(opType, part, subCol);
        }
        subCol += part.length;
      });
    }
  }

  function handleComment(content) {
    finishSub = finishComment;
    append(content);
  }

  function handleString(content) {
    if (content == quote) {
      changeHandler(handleCode, noop);
      quote = '';
    } else {
      append(content);
    }
  }

  function token(type, text, col) {
    tokens.push({type: type, text: text, pos: [lineNumber, col]});
  }

  function updateIndents(line) {
    var indent = line.match(/^\s*/)[0];
    if (indent) {
      line = line.substring(indent.length);
    }
    if (!line) {
      // Ignore lines of only whitespace.
      token(White, indent, 0);
    } else {
      var lastIndent = indents[indents.length - 1];
      if (startsWith(indent, lastIndent)) {
        if (indent.length === lastIndent.length) {
          // Same level.
          if (indent) token(White, indent, 0);
        } else {
          // Indented further than before.
          indents.push(indent);
          token(Indent, indent, 0);
        }
      } else {
        // Find where in the history we are.
        // TODO Or is it better to judge by 'end' tokens? I don't think so.
        var tokenContents = indent;
        while (indents.length > 1) {
          indents.pop();
          token(Dedent, tokenContents, 0);
          tokenContents = '';
          lastIndent = indents[indents.length - 1];
          if (lastIndent === indent) {
            break;
          }
        }
      }
    }
    colNumber += indent.length;
    return line;
  }

}


/**
 * Generic traversal support. Provides a canonical order.
 *
 * Only works for expr nodes right now, maybe?
 *
 * TODO If useful, convert other traversals to using this.
 */
function traverseNode(node, handler) {

  // Marker object for ending traversal.
  var done = {};
  var result;

  try {
    // TODO Does traverseExpr work for any node or just exprs?
    // TODO Define a traverseAny?
    traverseExpr(node);
  } catch (e) {
    if (e === done) {
      return result;
    } else {
      // It was a real error.
      throw e;
    }
  }
  
  function handle(node) {
    result = handler(node);
    if (result !== undefined) {
      throw done;
    }
  }

  // TODO Combine with traverseBlock as traverseExprs?
  function traverseArgs(node) {
    traverseBlock(node);
  }

  function traverseBlock(node) {
    each(node.expr, function(kid) {
      traverseExpr(kid);
    });
  }

  function traverseCall(node) {
    handle(node.id[0]);
    var args = node.args;
    for (var i = 0; i < args.length; i++) {
      traverseArgs(args[i]);
    }
  }

  function traverseDef(node) {
    handle(node.id[0]);
    var params = node.params[0];
    each(node.params[0].expr, function(param, p) {
      handle(param);
      handle(param.id);
    });
    // TODO Multidefs.
    traverseExpr(node.expr[0]);
  }

  function traverseExpr(node) {
    handle(node);
    switch(node.type) {
      case Block:
        traverseBlock(node);
        break;
      case Call:
        traverseCall(node);
        break;
      case Def:
        traverseDef(node);
        break;
      case Var:
        traverseVar(node);
        break;
      default:
    }
  }

  function traverseVar(node) {
    // TODO Merge some logic with traverseDef
    handle(node.id[0]);
    // TODO How to represent the 'put' side of things? -> put(it, @id, ...)
    if (node.expr) {
      traverseExpr(node.expr[0]);
    }
  }

}


})();
